{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "  return 3*x**2 - 4*x + 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f(3.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f9b98432ee0>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkRklEQVR4nO3dd3yV5f3/8dcnkwQCIRBmwg5hD4mIimhlSAXRWkXpF8E6UGtrbSutddTafq1YW7+ttQ6cqNRRxOLCURwIAhL2StgbQghkEMi+fn9w7I8qSNbJfcb7+XjwOOc+I/f76CNvLu5z3ddtzjlERCS0RHgdQERE6p/KXUQkBKncRURCkMpdRCQEqdxFREJQlNcBAFq2bOk6derkdQwRkaCybNmyg8655JM9FxDl3qlTJzIzM72OISISVMxsx6me02EZEZEQpHIXEQlBKncRkRCkchcRCUEqdxGREHTacjez58zsgJmtPeGxJDP7yMw2+W6bn/Dcr81ss5llm9lF/gouIiKnVp2R+wvA6K89dicwzzmXBszzbWNmvYCrgd6+9zxuZpH1llZERKrltOXunJsPHPraw5cCM3z3ZwCXnfD4q865UufcNmAzMLh+on5TTmEJv39nPYeLy/y1CxGRoFTbY+6tnXP7AHy3rXyPtwd2nfC63b7HvsHMpphZppll5ubm1ipE/tFynl2wjZcXn3Iev4hIWKrvL1TtJI+d9GogzrnpzrkM51xGcvJJz549rfQ2CVyQnsyMRdspKa+s1c8QEQlFtS33HDNrC+C7PeB7fDeQesLrUoC9tY93elOGdeHgkTLeWL7bn7sREQkqtS33t4DJvvuTgTknPH61mcWaWWcgDfiybhG/3dldWtAvpRnPfL6NyipdMlBEBKo3FfIVYBGQbma7zex6YBow0sw2ASN92zjn1gGvA+uB94FbnXN+PV5iZkwZ1oVtB4v5aH2OP3clIhI0TrsqpHNuwimeGn6K1z8APFCXUDU1uncbUpPimD5/C6P7tGnIXYuIBKSQOEM1KjKCG4Z2YfnOfDK3f33WpohI+AmJcge4MiOFxPhonpq/1esoIiKeC5lyj4+JYtKQjvx7Qw5bco94HUdExFMhU+4Ak87pRExkBM98rtG7iIS3kCr3lk1i+f6gFN5YtocDRSVexxER8UxIlTvAjed1obyqihlfbPc6ioiIZ0Ku3Du3bMyoXq15efFOiksrvI4jIuKJkCt3gCnDulJwrJzXlu46/YtFREJQSJb7oI7NObNTc55dsI2Kyiqv44iINLiQLHc4Pnrfk3+Md9fs8zqKiEiDC9lyH96jFV2TGzN9/lac04JiIhJeQrbcIyKOLyi2bm8hn2866HUcEZEGFbLlDnDZwPa0adqIxz7Z7HUUEZEGFdLlHhsVyZRhXfhy2yGWakExEQkjIV3uAFcPTiWpcQx/1+hdRMJIyJd7fEwU1w/tzKfZuazdU+B1HBGRBhHy5Q4wcUhHEmKjePxTjd5FJDyERbk3i4tm0jkdmbt2P5sPFHkdR0TE78Ki3AGuO7czsVERPPGplgMWkdAXNuXeokksEwZ34F8r97Dr0FGv44iI+FXYlDvAlGFdiDCYrkvxiUiIC6tyb9ssju+fkcJrmbs4UKiLeYhI6Aqrcge4+fyuVFRW8cyCbV5HERHxm7Ar904tG3NJ/3a8vHgH+UfLvI4jIuIXYVfuAD+6oBtHyyp5fuF2r6OIiPhFWJZ7epsERvZqzQtfbOeILsUnIiEoLMsd4NbvdKPgWDkzF+/wOoqISL0L23IfkJrI0G4tefrzbRwrq/Q6johIvQrbcge4bXgaB4+UMnOJRu8iElrCutwHd07i3G4tePKzLRq9i0hICetyB7h9RHcOHinjZR17F5EQEvblfmanJIZ2a8mTn23haJlmzohIaKhTuZvZz8xsnZmtNbNXzKyRmSWZ2Udmtsl327y+wvrLz0amkVdcxkuLNHoXkdBQ63I3s/bAbUCGc64PEAlcDdwJzHPOpQHzfNsBbVDHJM5La8lT87dq9C4iIaGuh2WigDgziwLigb3ApcAM3/MzgMvquI8GcfuI7hwqLuNFjd5FJATUutydc3uAPwE7gX1AgXPuQ6C1c26f7zX7gFYne7+ZTTGzTDPLzM3NrW2MejOoY3OGdU9m+vytFOusVREJcnU5LNOc46P0zkA7oLGZTazu+51z051zGc65jOTk5NrGqFc/G5Gm0buIhIS6HJYZAWxzzuU658qB2cA5QI6ZtQXw3R6oe8yGMbBDcy5IT2b6/C1ac0ZEglpdyn0nMMTM4s3MgOHABuAtYLLvNZOBOXWL2LBuH9Gdw0fLmfHFdq+jiIjUWl2OuS8BZgHLgTW+nzUdmAaMNLNNwEjfdtAYkJrId9KTefrzrRSVlHsdR0SkVuo0W8Y5d59zrodzro9z7hrnXKlzLs85N9w5l+a7PVRfYRvK7SO6k3+0XMfeRSRohf0ZqifTPzWR4T1aMX2+Ru8iEpxU7qfw0xFpFBwr5wVdrUlEgpDK/RT6pSQyomdrpn++VddaFZGgo3L/FlMvSudIaQVPfLbF6ygiIjWicv8W6W0SuGxAe15YuJ2cwhKv44iIVJvK/TR+NqI7Vc7x6LxNXkcREak2lftpdGgRz4TBHXht6S62Hyz2Oo6ISLWo3Kvhxxd2Izoygkc+2uh1FBGRalG5V0OrhEZcN7QTb63ay7q9BV7HERE5LZV7NU0Z1pVmcdH86YNsr6OIiJyWyr2amsVFc8sFXfkkO5cvtwXdigoiEmZU7jUw+exOtG4ayx/fz8I553UcEZFTUrnXQFxMJLcNTyNzx2E+yQ6aZepFJAyp3GtofEYqnVrE88f3s6mq0uhdRAKTyr2GoiMj+PmodLL2F/H26r1exxEROSmVey2M7duWXm2b8ucPN1JWUeV1HBGRb1C510JEhDF1dDo7Dx3ltaU7vY4jIvINKvdauqB7MkO6JPGXf2/SBT1EJOCo3GvJzLj74l7kFZfx+KdaElhEAovKvQ76pjTj8oHteXbBNnYfPup1HBGR/1C519EdF6VjwMNalkBEAojKvY7aJcZx43ldmLNyLyt35XsdR0QEULnXi5sv6ErLJrE88O56LUsgIgFB5V4PmsRG8fOR3Vm6/TAfrNvvdRwREZV7fRmfkUJaqyZMm5ulE5tExHMq93oSFRnBXWN6sj3vKC8t3uF1HBEJcyr3enRB92TOS2vJo/M2kX+0zOs4IhLGVO71yMy46+KeFJaU87ePN3sdR0TCmMq9nvVs25Txg1J5cdF2duQVex1HRMKUyt0PfjGqO9GRETz0fpbXUUQkTKnc/aBV00bcNKwr763Zz9Ltut6qiDQ8lbuf3DisM+2aNeK+Oeuo1BWbRKSB1anczSzRzGaZWZaZbTCzs80sycw+MrNNvtvm9RU2mMTHRHHXmJ6s31fIP77Umu8i0rDqOnL/K/C+c64H0B/YANwJzHPOpQHzfNthaUzftpzdpQV//jCbw8WaGikiDafW5W5mTYFhwLMAzrky51w+cCkww/eyGcBldYsYvMyM347rTVFJBX/6UKtGikjDqcvIvQuQCzxvZivM7Bkzawy0ds7tA/DdtjrZm81sipllmllmbm5uHWIEtvQ2CUw6uyP/+HIna/cUeB1HRMJEXco9CjgDeMI5NxAopgaHYJxz051zGc65jOTk5DrECHy3j+hOUnwM9721TqtGikiDqEu57wZ2O+eW+LZncbzsc8ysLYDv9kDdIga/ZnHR/Gp0D5btOMybK/Z4HUdEwkCty905tx/YZWbpvoeGA+uBt4DJvscmA3PqlDBEXDEohf6piTw4N0sX1BYRv6vrbJmfADPNbDUwAPgDMA0YaWabgJG+7bAXEWHcP643uUWlWndGRPwuqi5vds6tBDJO8tTwuvzcUDUgNZGrMlJ5bsE2xmek0q1VE68jiUiI0hmqDWzq6HTiYiK5/219uSoi/qNyb2Atm8Ty85Hd+XzTQT5Yl+N1HBEJUSp3D1wzpCPprRP4/TvrOVZW6XUcEQlBKncPREVGcP+lvdmTf4xHP97kdRwRCUEqd48M6dKCKwal8PT8rWTtL/Q6joiEGJW7h+6+uCdN46K5a/YaqrQssIjUI5W7h5o3juHui3uyfGe+lgUWCUOvL93Fsh2H/fKzVe4eu/yM9pzbrQUPvZ/FgcISr+OISAPZknuEe+as5bmF2/zy81XuHjMz/veyvpRWVHH/2+u9jiMiDaCqyvHr2WtoFBXBfZf08ss+VO4BoHPLxvzkO914d80+Ps7S3HeRUPfq0l18ue0Q94zpRauERn7Zh8o9QNx0fle6tWrCvf9ax9GyCq/jiIif5BSW8OB7GzinawuuzEjx235U7gEiJiqCBy/vy578Y/zl35r7LhKqfjNnLWWVVfzhe30xM7/tR+UeQM7slMSEwak8u2Ab6/bqqk0ioeb9tfv4YF0OPxvZnU4tG/t1Xyr3AHPn6J40jz8+971Sc99FQkbBsXLunbOO3u2acsPQzn7fn8o9wDSLj+besb1YtbuAlxZt9zqOiNSTaXM3cKi4jIe+34+oSP9Xr8o9AI3r345h3ZN5+INsdh066nUcEamjRVvyeOXLXdwwtDN92jdrkH2q3AOQmfGH7/UB4M7Zq7Xuu0gQKymv5K4319AhKZ7bR3RvsP2q3ANUSvN47h7Ti4Wb85i5REsTiASrR+dtYtvBYh68vC9xMZENtl+VewCbMDiV89Ja8of3NujwjEgQWr+3kKfmb+XKQSmc261lg+5b5R7AzIxp3+9HhBm/nLVaK0eKBJGyiiqmzlpF8/ho7h7Ts8H3r3IPcO0T47h7TE8Wbc1j5pIdXscRkWp67ONNrNtbyAPf60tifEyD71/lHgSuPvP44ZkH52bp8IxIEFi1K5+/f7qFy89oz0W923iSQeUeBE48PDN11iodnhEJYCXllfz89ZW0Sojlvkt6e5ZD5R4k2ifGcc+YnizeeoiXdXhGJGD98f1stuQW8/AV/WkWF+1ZDpV7ELnqzFSGdU/mwfey2JFX7HUcEfmaRVvyeG7hNiad3ZGhaQ07O+brVO5BxMx46Pt9iYowpmr2jEhAOVJawR3/XEWnFvHc+d0eXsdRuQebts3iuHdsL77cdogZWntGJGD87zvr2VdwjD+PH0B8TJTXcVTuwejKjBS+k57MtLlZbMwp8jqOSNj7OCuHV5fu4qbzuzKoY3Ov4wAq96BkZvzxiv40iY3itldWUFJe6XUkkbB1uLiMX72xhh5tErh9RJrXcf5D5R6kkhNiefjKfmTtL+Kh97O8jiMStu6ds5b8o2U8Mn4AsVENt3bM6ajcg9iFPVpz7TmdeH7hdj7NPuB1HJGwM2flHt5ZvY+fDk+jV7umXsf5Lyr3IHfnd3uQ3jqBO/65moNHSr2OIxI2duYd5e4313JGh0RuPr+r13G+oc7lbmaRZrbCzN7xbSeZ2Udmtsl3GxjfLoSoRtGRPDphIIUl5Uz95yqt/S7SAMoqqvjJK8uJMHh0wsAGubJSTdVHop8CG07YvhOY55xLA+b5tsWP0tskcPfFPfkkO5cXF+nsVRF/+/OH2azaXcAfr+hHSvN4r+OcVJ3K3cxSgDHAMyc8fCkww3d/BnBZXfYh1TPp7I5c2KMVD7y3gez9mh4p4i+fZh/gqflbmTikA6P7tPU6zinVdeT+F+CXQNUJj7V2zu0D8N22OtkbzWyKmWWaWWZubm4dY8jx6ZH9aNooWtMjRfzkQGEJv3h9FT3aJHDPmF5ex/lWtS53MxsLHHDOLavN+51z051zGc65jOTk5NrGkBO0bBLLn67sR3ZOEdPmanqkSH2qqnL87PWVFJdV8NgPBtIoOnCmPZ5MXUbu5wLjzGw78CpwoZm9DOSYWVsA363m6DWgC9Jbcd25nXnhi+18nJXjdRyRkPHEZ1tYuDmP+8f1plurBK/jnFaty90592vnXIpzrhNwNfCxc24i8BYw2feyycCcOqeUGvnVd9Pp1bYpP399FbsP6+IeInW1bMchHvloI5f0b8f4jFSv41SLP+bvTANGmtkmYKRvWxpQbFQkT0w8g8oqx49mLqe0QsffRWqr4Gg5t72yknaJjXjge30wM68jVUu9lLtz7lPn3Fjf/Tzn3HDnXJrv9lB97ENqpmOLxvz5yv6s3l3A795e73UckaDknOPO2avJKSzhbxPOoGkj7y6+UVOBN/Ne6s2o3m246fwuzFyykzdX7PY6jkjQeeGL7cxdu5+pF6UzIDXR6zg1onIPcVNHpXNW5yR+PXuN5r+L1MCSrXk88O4GRvRszY3ndfE6To2p3ENcVGQEf/vBQBIaRXPLy8soKin3OpJIwNtfUMKt/1hBh6R4HrmqPxERwXGc/UQq9zDQKqERj00YyI5DR/nlrNVaf0bkW5RWVHLLzGUcLavgqWsGBdVx9hOp3MPEWV1a8KvR6cxdu59nF2zzOo5IwPrd2+tZsTOfP13Zn7TWgT+f/VRU7mHkxvO6cFHv1kybm0Xmdk1iEvm615fuYuaSndx8flcu7hu468ZUh8o9jJgZD1/Zn5Tmcdz6j+XkFmn9d5GvrN6dzz1z1jK0W0vuGNXd6zh1pnIPM00bRfP4/wyi4Fg5N72UqQXGRIC8I6Xc/NIykpvEBuz67DUV/J9AaqxXu6Y8Mn4Ay3fmc9fsNfqCVcJaRWUVP3llBQeLy3hy4iCSGsd4HaleqNzD1MV92/Lzkd2ZvWIPT3621es4Ip754wfZfLEljz98ry99U5p5HafeRHkdQLzzkwu7senAEf74QRZdkxszqncbryOJNKhZy3Yz3XfhjSsGpXgdp15p5B7GzIyHr+hHv/bNuP21lazfW+h1JJEGs2hLHr+evZpzu7Xgvkt6ex2n3qncw1yj6EimT8qgaaNobpixVDNoJCxsyT3CzS8vo2OLxjz+P4OIDoEvUL8u9D6R1Fjrpo14elIGh46WaQaNhLxDxWVc98JSoiKM5689k2ZxwXkG6umo3AWAvinNNINGQl5JeSVTXsxkf0EJT0/OIDUp3utIfqNyl/+4uG9bfjbi+AyaJz7b4nUckXrlnOOXs1aTueMwj4wfwBkdmnsdya80W0b+y23Du7E59wgPf5BNSvN4xvVv53UkkXrxfx9t5K1Ve/nl6HTG9AvupQWqQ+Uu/+WrGTQ5BSX84vWVJMXHMDStpdexROrkjWW7efTjzVyVkcot53f1Ok6D0GEZ+YZG0ZE8PTmDrslNuOmlTFbvzvc6kkitLd6ax52zV3NO1xb8bxBdA7WuVO5yUs3ioplx3WAS42P44fNL2Xaw2OtIIjW2fm8hU17MpGOLxjwxMTSnPJ5K+HxSqbHWTRvx0vWDccA1zy7hQGGJ15FEqm3bwWImPbeExrFRvPDD0J3yeCoqd/lWXZKb8Ny1Z3KouIxJz31JoS7TJ0Fgb/4xJj6zhCoHL11/FinNQ3fK46mo3OW0BqQm8uTEQWw+cIQbZ+gkJwlseUdKmfjsEgqPlfPidYPp1qqJ15E8oXKXahnWPZk/j+/Pkm2H+OmrK6is0klOEngKS8qZ9NyX7Dl8jGevPZM+7UNnlceaUrlLtV06oD33ju3FB+tyuOdfOotVAsuxskpueCGT7P1FPHnNIAZ3TvI6kqc0z11q5Pqhnck7Usrjn24hOjKC+8f1DpupZRK4yiqquGXmMpbuOMTfJgzkO+mtvI7kOZW71NjUi9Ipr6zi6c+3YcBvVfDiocoqx89fX8mn2bk8eHlfxvbTWdWgcpdaMDPuurgnzsEzC7YBKnjxRmWV49ezV/PO6n3cdXEPJgzu4HWkgKFyl1oxM+4e0xMHPLtgG2bGfZf0UsFLg6morGLqrNW8uWIPtw1PY8qw8FhWoLpU7lJrZsY9Y3oCxwseUMFLgyivrOL2V1fy7pp93DGqOz++MM3rSAFH5S518lXBOwfPLdyGGfxmrApe/Ke0opIf/2MFH63P4e6Le3LjsC5eRwpIKnepMzPj3rE9cTieX7gdUMGLf5SUV3Lzy8v4NDuX313am0lnd/I6UsCqdbmbWSrwItAGqAKmO+f+amZJwGtAJ2A7MN45d7juUSWQmRm/GdsLgOcXbse54wUfEaGCl/pxtKyCG1/M5IsteTx4eV99eXoadTmJqQL4hXOuJzAEuNXMegF3AvOcc2nAPN+2hIGvCv6GoZ154Yvt3P7aSsoqqryOJSHgSGkF1z63lEVb8vjzlf1V7NVQ65G7c24fsM93v8jMNgDtgUuBC3wvmwF8CvyqTiklaHw1i6ZFk1geej+Lw0fLeGLiIJrE6gig1E7BsXKuff5LVu8u4NEJAzWPvZrqZfkBM+sEDASWAK19xf/VXwAnPVXMzKaYWaaZZebm5tZHDAkQZsYtF3Tl4Sv68cWWPCZMX8zBI6Vex5IgtCf/GFc++QVr9xTw+P+coWKvgTqXu5k1Ad4AbnfOFVb3fc656c65DOdcRnJycl1jSAC6MiOVpycNYtOBIq544gt25h31OpIEkbV7Crjs7wvZV1DCjB8O5qLebbyOFFTqVO5mFs3xYp/pnJvtezjHzNr6nm8LHKhbRAlmF/ZozT9uHEL+sXIuf+L4CEzkdD7JOsD4pxYRExnBG7ecwznddB3fmqp1udvxeW7PAhucc4+c8NRbwGTf/cnAnNrHk1BwRofmzLr5bGIijaunL+aLzQe9jiQBbOaSHdzwYiadWzbmzR+dQ/fWCV5HCkp1GbmfC1wDXGhmK31/LgamASPNbBMw0rctYa5bqwTe+NE5tEtsxLXPL2XOyj1eR5IAU1XlmDY3i7vfXMuwtJa8ftPZtGrayOtYQasus2UWAKeaxDy8tj9XQlfbZnH886ZzmPJSJj99dSXr9xbyy9E9iNRc+LBXUl7J1FmreXvVXn5wVgd+N643UWF0MWt/0H89aVDN4qN5+YazmHR2R56av5Vrn/+S/KNlXscSDx0qLmPSs1/y9qq93PndHjxwWR8Vez3Qf0FpcNGREfzu0j5Mu7wvS7YeYtxjC8naX+2JVhJClu88zJhHP2fl7nz+NmEgN5/fVctW1BOVu3jm6sEdeGXKEErKK7n88S+Yu2af15GkgTjneHHRdq56ahGREcbsW87hkv6aw16fVO7iqUEdm/P2T4aS3iaBW2Yu508fZFOli2+HtKNlFdz+2kp+M2cd56Ul8+5PzgvrC1n7i8pdPNe6aSNenTKEqzJSeeyTzdzwYiYFR8u9jiV+sPnAES59bCFvr9rL1IvSeWZSBs3io72OFZJU7hIQYqMimfb9vvz+sj7M35jLd/86n0Vb8ryOJfXo3dX7uPSxBeQVl/HidWdx63e6adVQP1K5S8AwM64Z0pE3bjmH2OhIfvDMYh6cu0ErSwa50opKfvf2em79x3LS2yTw7m1DGZqmM079TeUuAad/aiLv3jaUq8/swFOfbeV7jy9k84Eir2NJLazZXcAlf1vAcwu3ce05nXh1ytm0bRbndaywoHKXgBQfE8WDl/dl+jWD2FdQwphHF/DSou04py9bg0FZRRWPfJjNZY8vpOBYOc//8Ex+O643MVGqnIaiRbYloI3q3YYBHRKZ+s/V3DtnHZ9k5/LQ9/uRnBDrdTQ5hfV7C/nFP1exYV8hl5/RnvvG9taXph7QX6MS8FolNOKFH57Jby/pxYLNBxn9l/nMXr5bo/gAU15ZxaPzNjHusQXkFpXy9KQMHhk/QMXuEQuEX5CMjAyXmZnpdQwJAhtzipg6azWrduVzVuckfn9ZH60aGAA25hTxi9dXsWZPAeP6t+P+cb1p3jjG61ghz8yWOecyTvqcyl2CTVWV47XMXUybm0VxaQXXn9eZ2y5Mo7Eu5dfgCo6W85d5G3lp0Q6axkXzwGV9+G7ftl7HChvfVu76bZCgExFhTBjcgVG9WvPQ+1k89dlW3lq5l/su6cVFvdtobZIGUFFZxStLd/HIh9nkHyvn6jM7cMeo7rRoou9CAoVG7hL0Mrcf4p5/rSVrfxHnd0/m/nG96dSysdexQtbCzQf53dvryc4pYkiXJH4ztje92jX1OlZY0mEZCXkVlVXMWLSDRz7MpqyyivEZqfz4wm6aU12Pth8s5oH3NvDR+hxSmsdx98U9Gd1H/1LykspdwkZOYQmPztvE65m7MIwfnNWBH13QVVf0qYP9BSVMn7+VlxfvICrSuPU73bh+aGcaRUd6HS3sqdwl7Ow6dJTHPt7MrOW7iYo4vqzBzRd0paWOCVfb9oPFPDV/C7OW7abKweUD2zP1onT9RRlAVO4StnbkFfPXeZv414o9xEZFMvmcTkwZ1oUkTdM7paz9hTz+yRbeWb2XqMgIrspIZcqwLqQmxXsdTb5G5S5hb0vuEf767028vXov0ZERjO3XlolDOjIwNVHHjH2W7zzM459s5t8bDtA4JpKJZ3fk+qGdaZWgkXqgUrmL+GzKKWLGou28uXwPxWWV9G7XlIlDOnLpgHbEx4TfzOCCo+W8tXovs5btZtWufBLjo7nu3M5MPruTziwNAip3ka85UlrBmyv2MHPxDrL2F5EQG8XlZ7Rn4pCOpIX4Ga+VVY6Fmw/yz2W7+WDdfsoqqujRJoHxGalcdWaqTgYLIip3kVNwzrFsx2FeWryDuWv2U1ZZRb+UZozq1ZpRvduQ1qpJyBy22XawmFnLdjF7+R72FZTQLC6aywa044pBqfRp3zRkPmc4UbmLVEPekVJmLdvN3LX7WbkrH4BOLeIZ1bsNo3q1ZmCH5kQG0ZWDSsor+XLbIT7bmMv8jblsOnCECINh3ZO5clAqI3q1IjZK0xmDmcpdpIZyCkv4aH0OH67PYdGWg5RXOlo2iWF4j9YM6ZpE/5REOrdsHFCjXeccW3KLmb8xl8825rJ4ax6lFVXEREVwVuckzu+ezNh+7WjTTF+QhgqVu0gdFJaU82l2Lh+u289n2bkUlVYA0Cwumv6piQxIacaADon0T0lssLVVKqscO/KK2bCviKz9hWzYV8S6vQXsKygBoEtyY4alJXN+ejJDOrcgLkYj9FCkchepJ5VVjk0Hili5M5+Vu47/2ZhTRJXv16h9YhypSXG0S4yjXbPjt20TG9E+MY62zRqR0Kh6M1COllWQd6SMQ8XH/+QVl3GouJQtB4rJ2l9Idk4RJeXHry0bGWF0admYHm2bMqRLEsPSkjUnPUyo3EX8qLi0gjV7Cli5K591ewvZm3+MffnHyCkqpbLqv3+/YqMiiI6MICrSiIowIiOMqIgI361RWlFFXnHpf4r765Iax9CzbQI92jSlR5sEerZtSrdWTbQUQJjSkr8iftQ4NoohXVowpEuL/3q8orKKA0Wl7Cs4xt78EvbmHyOvuIyKSkdlVRUVVY6KSkdF1f/fjomMIKlxDElNYmjROIakxrEkNfbdbxJDQmxUQB3nl8Clchfxk6jIiOOHZxLjGNTR6zQSbnQNVRGREKRyFxEJQX4rdzMbbWbZZrbZzO70135EROSb/FLuZhYJ/B34LtALmGBmvfyxLxER+SZ/jdwHA5udc1udc2XAq8ClftqXiIh8jb/KvT2w64Tt3b7H/sPMpphZppll5ubm+imGiEh48le5n2wi7n+dzeGcm+6cy3DOZSQnJ/sphohIePJXue8GUk/YTgH2+mlfIiLyNX5ZfsDMooCNwHBgD7AU+IFzbt0pXp8L7Kj3IA2jJXDQ6xAeCNfPDeH72fW5A09H59xJD3345QxV51yFmf0Y+ACIBJ47VbH7Xh+0x2XMLPNUazuEsnD93BC+n12fO7j4bfkB59x7wHv++vkiInJqOkNVRCQEqdzrbrrXATwSrp8bwvez63MHkYBYz11EROqXRu4iIiFI5S4iEoJU7vXEzO4wM2dmLb3O0lDM7GEzyzKz1Wb2ppklep3Jn8J1pVMzSzWzT8xsg5mtM7Ofep2pIZlZpJmtMLN3vM5SEyr3emBmqcBIYKfXWRrYR0Af51w/jp+09muP8/hNmK90WgH8wjnXExgC3BpGnx3gp8AGr0PUlMq9fvwf8Eu+tn5OqHPOfeicq/BtLub4MhOhKmxXOnXO7XPOLffdL+J40bX/9neFBjNLAcYAz3idpaZU7nVkZuOAPc65VV5n8dh1wFyvQ/jRaVc6DQdm1gkYCCzxOEpD+QvHB25VHueoMV0guxrM7N9Am5M8dTdwFzCqYRM1nG/77M65Ob7X3M3xf7rPbMhsDey0K52GOjNrArwB3O6cK/Q6j7+Z2VjggHNumZld4HGcGlO5V4NzbsTJHjezvkBnYJWZwfHDEsvNbLBzbn8DRvSbU332r5jZZGAsMNyF9kkTYb3SqZlFc7zYZzrnZnudp4GcC4wzs4uBRkBTM3vZOTfR41zVopOY6pGZbQcynHOBuoJcvTKz0cAjwPnOuZC+4kpNVzoNJXZ85DIDOOScu93jOJ7wjdzvcM6N9ThKtemYu9TFY0AC8JGZrTSzJ70O5C++L46/Wul0A/B6OBS7z7nANcCFvv/PK32jWQlgGrmLiIQgjdxFREKQyl1EJASp3EVEQpDKXUQkBKncRURCkMpdRCQEqdxFRELQ/wOVnmci5YAbRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = np.arange(-5, 5, 0.25)\n",
    "ys = f(xs)\n",
    "plt.plot(xs, ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.999378523327323e-06"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = 0.000001\n",
    "x = 2/3\n",
    "(f(x + h) - f(x))/h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.0\n"
     ]
    }
   ],
   "source": [
    "# les get more complex\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "d = a*b + c\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d1 4.0\n",
      "d2 4.0001\n",
      "slope 0.9999999999976694\n"
     ]
    }
   ],
   "source": [
    "h = 0.0001\n",
    "\n",
    "# inputs\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "\n",
    "d1 = a*b + c\n",
    "c += h\n",
    "d2 = a*b + c\n",
    "\n",
    "print('d1', d1)\n",
    "print('d2', d2)\n",
    "print('slope', (d2 - d1)/h)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "  \n",
    "  def __init__(self, data, _children=(), _op='', label=''):\n",
    "    self.data = data\n",
    "    self.grad = 0.0\n",
    "    self._backward = lambda: None\n",
    "    self._prev = set(_children)\n",
    "    self._op = _op\n",
    "    self.label = label\n",
    "\n",
    "  def __repr__(self):\n",
    "    return f\"Value(data={self.data})\"\n",
    "  \n",
    "  def __add__(self, other):\n",
    "    other = other if isinstance(other, Value) else Value(other)\n",
    "    out = Value(self.data + other.data, (self, other), '+')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += 1.0 * out.grad\n",
    "      other.grad += 1.0 * out.grad\n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "\n",
    "  def __mul__(self, other):\n",
    "    other = other if isinstance(other, Value) else Value(other)\n",
    "    out = Value(self.data * other.data, (self, other), '*')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += other.data * out.grad\n",
    "      other.grad += self.data * out.grad\n",
    "    out._backward = _backward\n",
    "      \n",
    "    return out\n",
    "  \n",
    "  def __pow__(self, other):\n",
    "    assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
    "    out = Value(self.data**other, (self,), f'**{other}')\n",
    "\n",
    "    def _backward():\n",
    "        self.grad += other * (self.data ** (other - 1)) * out.grad\n",
    "    out._backward = _backward\n",
    "\n",
    "    return out\n",
    "  \n",
    "  def __rmul__(self, other): # other * self\n",
    "    return self * other\n",
    "\n",
    "  def __truediv__(self, other): # self / other\n",
    "    return self * other**-1\n",
    "\n",
    "  def __neg__(self): # -self\n",
    "    return self * -1\n",
    "\n",
    "  def __sub__(self, other): # self - other\n",
    "    return self + (-other)\n",
    "\n",
    "  def __radd__(self, other): # other + self\n",
    "    return self + other\n",
    "\n",
    "  def tanh(self):\n",
    "    x = self.data\n",
    "    t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "    out = Value(t, (self, ), 'tanh')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += (1 - t**2) * out.grad\n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "  \n",
    "  def exp(self):\n",
    "    x = self.data\n",
    "    out = Value(math.exp(x), (self, ), 'exp')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += out.data * out.grad # NOTE: in the video I incorrectly used = instead of +=. Fixed here.\n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "  \n",
    "  \n",
    "  def backward(self):\n",
    "    \n",
    "    topo = []\n",
    "    visited = set()\n",
    "    def build_topo(v):\n",
    "      if v not in visited:\n",
    "        visited.add(v)\n",
    "        for child in v._prev:\n",
    "          build_topo(child)\n",
    "        topo.append(v)\n",
    "    build_topo(self)\n",
    "    \n",
    "    self.grad = 1.0\n",
    "    for node in reversed(topo):\n",
    "      node._backward()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "  # builds a set of all nodes and edges in a graph\n",
    "  nodes, edges = set(), set()\n",
    "  def build(v):\n",
    "    if v not in nodes:\n",
    "      nodes.add(v)\n",
    "      for child in v._prev:\n",
    "        edges.add((child, v))\n",
    "        build(child)\n",
    "  build(root)\n",
    "  return nodes, edges\n",
    "\n",
    "def draw_dot(root):\n",
    "  dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n",
    "  \n",
    "  nodes, edges = trace(root)\n",
    "  for n in nodes:\n",
    "    uid = str(id(n))\n",
    "    # for any value in the graph, create a rectangular ('record') node for it\n",
    "    dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
    "    if n._op:\n",
    "      # if this value is a result of some operation, create an op node for it\n",
    "      dot.node(name = uid + n._op, label = n._op)\n",
    "      # and connect this node to it\n",
    "      dot.edge(uid + n._op, uid)\n",
    "\n",
    "  for n1, n2 in edges:\n",
    "    # connect n1 to the op node of n2\n",
    "    dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "\n",
    "  return dot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs x1,x2\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "# weights w1,w2\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "# bias of the neuron\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "# x1*w1 + x2*w2 + b\n",
    "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label = 'n'\n",
    "o = n.tanh(); o.label = 'o'\n",
    "o.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (20220226.1711)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1575pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1575.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1571,-206 1571,4 -4,4\"/>\n",
       "<!-- 140307325553680 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140307325553680</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-55.5 326,-91.5 546,-91.5 546,-55.5 326,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-55.5 379,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-55.5 464,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307325551664+ -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140307325551664+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140307325553680&#45;&gt;140307325551664+ -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140307325553680&#45;&gt;140307325551664+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-90.75C555.64,-92.23 564.6,-93.65 572.65,-94.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.23,-98.4 582.65,-96.5 573.32,-91.48 572.23,-98.4\"/>\n",
       "</g>\n",
       "<!-- 140307325553680* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140307325553680*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140307325553680*&#45;&gt;140307325553680 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140307325553680*&#45;&gt;140307325553680</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C297.77,-73.5 306.37,-73.5 315.6,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-77 325.84,-73.5 315.84,-70 315.84,-77\"/>\n",
       "</g>\n",
       "<!-- 140306250485280 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140306250485280</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 200,-91.5 200,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-55.5 33,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-55.5 118,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140306250485280&#45;&gt;140307325553680* -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140306250485280&#45;&gt;140307325553680*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200.21,-73.5C209.2,-73.5 217.86,-73.5 225.7,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.85,-77 235.85,-73.5 225.85,-70 225.85,-77\"/>\n",
       "</g>\n",
       "<!-- 140307325551664 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140307325551664</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307325551424+ -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140307325551424+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140307325551664&#45;&gt;140307325551424+ -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140307325551664&#45;&gt;140307325551424+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140307325551664+&#45;&gt;140307325551664 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140307325551664+&#45;&gt;140307325551664</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "<!-- 140307325553728 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140307325553728</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-109.5 1382,-145.5 1567,-145.5 1567,-109.5 1382,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1393.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1405,-109.5 1405,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1485,-109.5 1485,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1526\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140307325553728tanh -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140307325553728tanh</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- 140307325553728tanh&#45;&gt;140307325553728 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140307325553728tanh&#45;&gt;140307325553728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.04,-127.5C1353.58,-127.5 1362.3,-127.5 1371.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.81,-131 1381.81,-127.5 1371.81,-124 1371.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140306786257456 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140306786257456</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-165.5 2.5,-201.5 197.5,-201.5 197.5,-165.5 2.5,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-165.5 35.5,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-165.5 115.5,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140307325550800* -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140307325550800*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140306786257456&#45;&gt;140307325550800* -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140306786257456&#45;&gt;140307325550800*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140307325550800 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140307325550800</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-110.5 328.5,-146.5 543.5,-146.5 543.5,-110.5 328.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-110.5 381.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-110.5 461.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307325550800&#45;&gt;140307325551664+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140307325550800&#45;&gt;140307325551664+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-111.01C554.01,-109.34 563.76,-107.74 572.44,-106.32\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.17,-109.75 582.47,-104.68 572.04,-102.84 573.17,-109.75\"/>\n",
       "</g>\n",
       "<!-- 140307325550800*&#45;&gt;140307325550800 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140307325550800*&#45;&gt;140307325550800</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C298.51,-128.5 308.08,-128.5 318.36,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-132 328.39,-128.5 318.39,-125 318.39,-132\"/>\n",
       "</g>\n",
       "<!-- 140306250483984 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140306250483984</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140306250483984&#45;&gt;140307325551424+ -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140306250483984&#45;&gt;140307325551424+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140306786256688 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140306786256688</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-0.5 2,-36.5 198,-36.5 198,-0.5 2,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-0.5 32,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-0.5 112,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5000</text>\n",
       "</g>\n",
       "<!-- 140306786256688&#45;&gt;140307325553680* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140306786256688&#45;&gt;140307325553680*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.28,-36.5C179.65,-39.61 190.16,-42.98 200,-46.5 210.28,-50.17 221.28,-54.74 231.11,-59.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"229.93,-62.37 240.48,-63.27 232.79,-55.99 229.93,-62.37\"/>\n",
       "</g>\n",
       "<!-- 140307325551424 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140307325551424</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307325551424&#45;&gt;140307325553728tanh -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140307325551424&#45;&gt;140307325553728tanh</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-127.5C1265.01,-127.5 1273.74,-127.5 1281.66,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1281.91,-131 1291.91,-127.5 1281.91,-124 1281.91,-131\"/>\n",
       "</g>\n",
       "<!-- 140307325551424+&#45;&gt;140307325551424 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140307325551424+&#45;&gt;140307325551424</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140306786257264 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140306786257264</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-110.5 4,-146.5 196,-146.5 196,-110.5 4,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-110.5 34,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-110.5 114,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140306786257264&#45;&gt;140307325550800* -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140306786257264&#45;&gt;140307325550800*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.05,-128.5C206.52,-128.5 216.65,-128.5 225.71,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.79,-132 235.79,-128.5 225.79,-125 225.79,-132\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f9bc84d8d30>"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 3.0.0 (20220226.1711)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"2944pt\" height=\"236pt\"\n",
       " viewBox=\"0.00 0.00 2944.00 236.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 232)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-232 2940,-232 2940,4 -4,4\"/>\n",
       "<!-- 140306525507104 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140306525507104</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2225,-108.5 2225,-144.5 2407,-144.5 2407,-108.5 2225,-108.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2235\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2245,-108.5 2245,-144.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2285\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 4.8284</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2325,-108.5 2325,-144.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2366\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.1464</text>\n",
       "</g>\n",
       "<!-- 140307325465216* -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140307325465216*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2688\" cy=\"-153.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2688\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140306525507104&#45;&gt;140307325465216* -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140306525507104&#45;&gt;140307325465216*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2407.04,-133.07C2485.41,-138.79 2594.98,-146.79 2650.96,-150.87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2650.87,-154.37 2661.1,-151.61 2651.38,-147.39 2650.87,-154.37\"/>\n",
       "</g>\n",
       "<!-- 140306525507104+ -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140306525507104+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1940\" cy=\"-126.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1940\" y=\"-122.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140306525507104+&#45;&gt;140306525507104 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140306525507104+&#45;&gt;140306525507104</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1967,-126.5C2017.46,-126.5 2131.06,-126.5 2214.81,-126.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2214.87,-130 2224.87,-126.5 2214.87,-123 2214.87,-130\"/>\n",
       "</g>\n",
       "<!-- 140307325464640 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140307325464640</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2443,-161.5 2443,-197.5 2625,-197.5 2625,-161.5 2443,-161.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2453\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2463,-161.5 2463,-197.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2503\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.1464</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2543,-161.5 2543,-197.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2584\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.8284</text>\n",
       "</g>\n",
       "<!-- 140307325464640&#45;&gt;140307325465216* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140307325464640&#45;&gt;140307325465216*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2625.18,-164.08C2634.42,-162.49 2643.38,-160.96 2651.47,-159.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2652.29,-162.99 2661.56,-157.85 2651.11,-156.09 2652.29,-162.99\"/>\n",
       "</g>\n",
       "<!-- 140307325464640**&#45;1 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140307325464640**&#45;1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"2316\" cy=\"-181.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2316\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">**&#45;1</text>\n",
       "</g>\n",
       "<!-- 140307325464640**&#45;1&#45;&gt;140307325464640 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140307325464640**&#45;1&#45;&gt;140307325464640</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2343.05,-181.26C2365.53,-181.05 2399.48,-180.74 2432.5,-180.43\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2432.74,-183.93 2442.7,-180.34 2432.67,-176.93 2432.74,-183.93\"/>\n",
       "</g>\n",
       "<!-- 140307057181792 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140307057181792</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"328.5,-110.5 328.5,-146.5 543.5,-146.5 543.5,-110.5 328.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"381.5,-110.5 381.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"461.5,-110.5 461.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"502.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307057184240+ -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>140307057184240+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"609\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"609\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140307057181792&#45;&gt;140307057184240+ -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140307057181792&#45;&gt;140307057184240+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M543.84,-111.01C554.01,-109.34 563.76,-107.74 572.44,-106.32\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.17,-109.75 582.47,-104.68 572.04,-102.84 573.17,-109.75\"/>\n",
       "</g>\n",
       "<!-- 140307057181792* -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140307057181792*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140307057181792*&#45;&gt;140307057181792 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140307057181792*&#45;&gt;140307057181792</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-128.5C298.51,-128.5 308.08,-128.5 318.36,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"318.39,-132 328.39,-128.5 318.39,-125 318.39,-132\"/>\n",
       "</g>\n",
       "<!-- 140307325465216 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140307325465216</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2751,-135.5 2751,-171.5 2936,-171.5 2936,-135.5 2751,-135.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2762.5\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">o</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2774,-135.5 2774,-171.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2814\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.7071</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2854,-135.5 2854,-171.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2895\" y=\"-149.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140307325465216*&#45;&gt;140307325465216 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140307325465216*&#45;&gt;140307325465216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2715.04,-153.5C2722.58,-153.5 2731.3,-153.5 2740.57,-153.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2740.81,-157 2750.81,-153.5 2740.81,-150 2740.81,-157\"/>\n",
       "</g>\n",
       "<!-- 140306786144896 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140306786144896</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"716,-137.5 716,-173.5 901,-173.5 901,-137.5 716,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"739,-137.5 739,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"779\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"819,-137.5 819,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307057183664+ -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140307057183664+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1008\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140306786144896&#45;&gt;140307057183664+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140306786144896&#45;&gt;140307057183664+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M901.02,-142.52C926,-138.98 951.59,-135.36 971.37,-132.55\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"972.03,-135.99 981.43,-131.12 971.04,-129.06 972.03,-135.99\"/>\n",
       "</g>\n",
       "<!-- 140307325467808 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140307325467808</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1690.5,-191.5 1690.5,-227.5 1876.5,-227.5 1876.5,-191.5 1690.5,-191.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1700.5\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1710.5,-191.5 1710.5,-227.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1750.5\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1790.5,-191.5 1790.5,-227.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1833.5\" y=\"-205.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.1036</text>\n",
       "</g>\n",
       "<!-- 140307325466992+ -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140307325466992+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1940\" cy=\"-181.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1940\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 140307325467808&#45;&gt;140307325466992+ -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140307325467808&#45;&gt;140307325466992+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1876.6,-192.81C1886.15,-191.08 1895.37,-189.41 1903.68,-187.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1904.43,-191.32 1913.65,-186.09 1903.18,-184.43 1904.43,-191.32\"/>\n",
       "</g>\n",
       "<!-- 140306786145184 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140306786145184</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2,-55.5 2,-91.5 198,-91.5 198,-55.5 2,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"17\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"32,-55.5 32,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"112,-55.5 112,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;1.5000</text>\n",
       "</g>\n",
       "<!-- 140306786146720* -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140306786146720*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"263\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140306786145184&#45;&gt;140306786146720* -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140306786145184&#45;&gt;140306786146720*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.37,-73.5C208.05,-73.5 217.4,-73.5 225.8,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.91,-77 235.91,-73.5 225.91,-70 225.91,-77\"/>\n",
       "</g>\n",
       "<!-- 140306786147536 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140306786147536</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 200,-36.5 200,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"16.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"33,-0.5 33,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118,-0.5 118,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 140306786147536&#45;&gt;140306786146720* -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140306786147536&#45;&gt;140306786146720*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-36.56C181.84,-39.33 191.2,-42.33 200,-45.5 210.53,-49.29 221.75,-54.1 231.72,-58.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"230.28,-61.86 240.82,-62.93 233.25,-55.52 230.28,-61.86\"/>\n",
       "</g>\n",
       "<!-- 140306525528800 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140306525528800</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1692.5,-136.5 1692.5,-172.5 1874.5,-172.5 1874.5,-136.5 1692.5,-136.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1702.5\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1712.5,-136.5 1712.5,-172.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1752.5\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 5.8284</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1792.5,-136.5 1792.5,-172.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1833.5\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0429</text>\n",
       "</g>\n",
       "<!-- 140306525528800&#45;&gt;140306525507104+ -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140306525528800&#45;&gt;140306525507104+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1874.82,-138.13C1885.07,-136.27 1895.01,-134.47 1903.88,-132.87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1904.54,-136.3 1913.75,-131.08 1903.29,-129.42 1904.54,-136.3\"/>\n",
       "</g>\n",
       "<!-- 140306525528800&#45;&gt;140307325466992+ -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140306525528800&#45;&gt;140307325466992+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1874.82,-170.28C1885.07,-172.08 1895.01,-173.81 1903.88,-175.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1903.3,-178.81 1913.75,-177.09 1904.5,-171.92 1903.3,-178.81\"/>\n",
       "</g>\n",
       "<!-- 140306525528800exp -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140306525528800exp</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1627\" cy=\"-154.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1627\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">exp</text>\n",
       "</g>\n",
       "<!-- 140306525528800exp&#45;&gt;140306525528800 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140306525528800exp&#45;&gt;140306525528800</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1654.21,-154.5C1662.34,-154.5 1671.85,-154.5 1681.94,-154.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1682.19,-158 1692.19,-154.5 1682.19,-151 1682.19,-158\"/>\n",
       "</g>\n",
       "<!-- 140306786146720 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140306786146720</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"326,-55.5 326,-91.5 546,-91.5 546,-55.5 326,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"352.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"379,-55.5 379,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"464,-55.5 464,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"505\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140306786146720&#45;&gt;140307057184240+ -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140306786146720&#45;&gt;140307057184240+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546.27,-90.75C555.64,-92.23 564.6,-93.65 572.65,-94.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"572.23,-98.4 582.65,-96.5 573.32,-91.48 572.23,-98.4\"/>\n",
       "</g>\n",
       "<!-- 140306786146720*&#45;&gt;140306786146720 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140306786146720*&#45;&gt;140306786146720</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M290.34,-73.5C297.77,-73.5 306.37,-73.5 315.6,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.84,-77 325.84,-73.5 315.84,-70 315.84,-77\"/>\n",
       "</g>\n",
       "<!-- 140307057180928 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140307057180928</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1072.5,-164.5 1072.5,-200.5 1254.5,-200.5 1254.5,-164.5 1072.5,-164.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-178.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1092.5,-164.5 1092.5,-200.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1132.5\" y=\"-178.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1172.5,-164.5 1172.5,-200.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1213.5\" y=\"-178.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.2203</text>\n",
       "</g>\n",
       "<!-- 140307057184096* -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140307057184096*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-154.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 140307057180928&#45;&gt;140307057184096* -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>140307057180928&#45;&gt;140307057184096*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1254.68,-166.05C1264.61,-164.24 1274.23,-162.48 1282.84,-160.91\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1283.6,-164.33 1292.81,-159.09 1282.35,-157.45 1283.6,-164.33\"/>\n",
       "</g>\n",
       "<!-- 140306786145712 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140306786145712</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"4,-165.5 4,-201.5 196,-201.5 196,-165.5 4,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">x2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"34,-165.5 34,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"114,-165.5 114,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140306786145712&#45;&gt;140307057181792* -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140306786145712&#45;&gt;140307057181792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.53,-165.44C181.84,-162.67 191.2,-159.67 200,-156.5 210.53,-152.71 221.75,-147.9 231.72,-143.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.25,-146.48 240.82,-139.07 230.28,-140.14 233.25,-146.48\"/>\n",
       "</g>\n",
       "<!-- 140307057184096 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140307057184096</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1382,-136.5 1382,-172.5 1564,-172.5 1564,-136.5 1382,-136.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1392\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1402,-136.5 1402,-172.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1442\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.7627</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1482,-136.5 1482,-172.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1523\" y=\"-150.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.2500</text>\n",
       "</g>\n",
       "<!-- 140307057184096&#45;&gt;140306525528800exp -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140307057184096&#45;&gt;140306525528800exp</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1564.18,-154.5C1573.14,-154.5 1581.83,-154.5 1589.73,-154.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1589.96,-158 1599.96,-154.5 1589.96,-151 1589.96,-158\"/>\n",
       "</g>\n",
       "<!-- 140307057184096*&#45;&gt;140307057184096 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140307057184096*&#45;&gt;140307057184096</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1346.13,-154.5C1353.67,-154.5 1362.39,-154.5 1371.64,-154.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1371.87,-158 1381.87,-154.5 1371.87,-151 1371.87,-158\"/>\n",
       "</g>\n",
       "<!-- 140307325466992 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140307325466992</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2003,-163.5 2003,-199.5 2189,-199.5 2189,-163.5 2003,-163.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"2013\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2023,-163.5 2023,-199.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2063\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 6.8284</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2103,-163.5 2103,-199.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"2146\" y=\"-177.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad &#45;0.1036</text>\n",
       "</g>\n",
       "<!-- 140307325466992&#45;&gt;140307325464640**&#45;1 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140307325466992&#45;&gt;140307325464640**&#45;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M2189.14,-181.5C2220.72,-181.5 2254.24,-181.5 2278.74,-181.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"2278.98,-185 2288.98,-181.5 2278.98,-178 2278.98,-185\"/>\n",
       "</g>\n",
       "<!-- 140307325466992+&#45;&gt;140307325466992 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140307325466992+&#45;&gt;140307325466992</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1967.12,-181.5C1974.62,-181.5 1983.29,-181.5 1992.5,-181.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1992.69,-185 2002.69,-181.5 1992.69,-178 1992.69,-185\"/>\n",
       "</g>\n",
       "<!-- 140306525505952 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140306525505952</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1690,-81.5 1690,-117.5 1877,-117.5 1877,-81.5 1690,-81.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1700\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\"> </text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1710,-81.5 1710,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1752.5\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1795,-81.5 1795,-117.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1836\" y=\"-95.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.1464</text>\n",
       "</g>\n",
       "<!-- 140306525505952&#45;&gt;140306525507104+ -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140306525505952&#45;&gt;140306525507104+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1877.05,-115.67C1886.34,-117.3 1895.33,-118.87 1903.44,-120.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1903.1,-123.78 1913.55,-122.05 1904.3,-116.88 1903.1,-123.78\"/>\n",
       "</g>\n",
       "<!-- 140307057183664 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140307057183664</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1071,-109.5 1071,-145.5 1256,-145.5 1256,-109.5 1071,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1082.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">n</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1094,-109.5 1094,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1134\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 0.8814</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1174,-109.5 1174,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1215\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307057183664&#45;&gt;140307057184096* -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140307057183664&#45;&gt;140307057184096*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1256.01,-143.6C1265.49,-145.26 1274.66,-146.88 1282.9,-148.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1282.36,-151.78 1292.81,-150.07 1283.57,-144.89 1282.36,-151.78\"/>\n",
       "</g>\n",
       "<!-- 140307057183664+&#45;&gt;140307057183664 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140307057183664+&#45;&gt;140307057183664</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1035.04,-127.5C1042.58,-127.5 1051.3,-127.5 1060.57,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1060.81,-131 1070.81,-127.5 1060.81,-124 1060.81,-131\"/>\n",
       "</g>\n",
       "<!-- 140306786145232 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>140306786145232</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-110.5 2.5,-146.5 197.5,-146.5 197.5,-110.5 2.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"19\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-110.5 35.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"75.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"115.5,-110.5 115.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.0000</text>\n",
       "</g>\n",
       "<!-- 140306786145232&#45;&gt;140307057181792* -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140306786145232&#45;&gt;140307057181792*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M197.91,-128.5C207.65,-128.5 217.05,-128.5 225.52,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.7,-132 235.7,-128.5 225.7,-125 225.7,-132\"/>\n",
       "</g>\n",
       "<!-- 140307057184240 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>140307057184240</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"672,-82.5 672,-118.5 945,-118.5 945,-82.5 672,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"725\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">x1*w1 + x2*w2</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"778,-82.5 778,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"863,-82.5 863,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 140307057184240&#45;&gt;140307057183664+ -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140307057184240&#45;&gt;140307057183664+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M941.24,-118.51C952.12,-120 962.4,-121.4 971.45,-122.64\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"971.11,-126.13 981.49,-124.01 972.05,-119.19 971.11,-126.13\"/>\n",
       "</g>\n",
       "<!-- 140307057184240+&#45;&gt;140307057184240 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140307057184240+&#45;&gt;140307057184240</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M636.23,-100.5C643.7,-100.5 652.41,-100.5 661.87,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"661.98,-104 671.98,-100.5 661.98,-97 661.98,-104\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f9ba89d2370>"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# inputs x1,x2\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "# weights w1,w2\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "# bias of the neuron\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "# x1*w1 + x2*w2 + b\n",
    "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + b; n.label = 'n'\n",
    "# ----\n",
    "e = (2*n).exp()\n",
    "o = (e - 1) / (e + 1)\n",
    "# ----\n",
    "o.label = 'o'\n",
    "o.backward()\n",
    "draw_dot(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7071066904050358\n",
      "---\n",
      "x2 0.5000001283844369\n",
      "w2 0.0\n",
      "x1 -1.5000003851533106\n",
      "w1 1.0000002567688737\n"
     ]
    }
   ],
   "source": [
    "\n",
    "x1 = torch.Tensor([2.0]).double()                ; x1.requires_grad = True\n",
    "x2 = torch.Tensor([0.0]).double()                ; x2.requires_grad = True\n",
    "w1 = torch.Tensor([-3.0]).double()               ; w1.requires_grad = True\n",
    "w2 = torch.Tensor([1.0]).double()                ; w2.requires_grad = True\n",
    "b = torch.Tensor([6.8813735870195432]).double()  ; b.requires_grad = True\n",
    "n = x1*w1 + x2*w2 + b\n",
    "o = torch.tanh(n)\n",
    "\n",
    "print(o.data.item())\n",
    "o.backward()\n",
    "\n",
    "print('---')\n",
    "print('x2', x2.grad.item())\n",
    "print('w2', w2.grad.item())\n",
    "print('x1', x1.grad.item())\n",
    "print('w1', w1.grad.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Neuron:\n",
    "  \n",
    "  def __init__(self, nin):\n",
    "    self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "    self.b = Value(random.uniform(-1,1))\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    # w * x + b\n",
    "    act = sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
    "    out = act.tanh()\n",
    "    return out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return self.w + [self.b]\n",
    "\n",
    "class Layer:\n",
    "  \n",
    "  def __init__(self, nin, nout):\n",
    "    self.neurons = [Neuron(nin) for _ in range(nout)]\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    outs = [n(x) for n in self.neurons]\n",
    "    return outs[0] if len(outs) == 1 else outs\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [p for neuron in self.neurons for p in neuron.parameters()]\n",
    "\n",
    "class MLP:\n",
    "  \n",
    "  def __init__(self, nin, nouts):\n",
    "    sz = [nin] + nouts\n",
    "    self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    for layer in self.layers:\n",
    "      x = layer(x)\n",
    "    return x\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [p for layer in self.layers for p in layer.parameters()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 665,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value(data=0.16578526021381612)"
      ]
     },
     "execution_count": 665,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "n(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 666,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [\n",
    "  [2.0, 3.0, -1.0],\n",
    "  [3.0, -1.0, 0.5],\n",
    "  [0.5, 1.0, 1.0],\n",
    "  [1.0, 1.0, -1.0],\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0] # desired targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 682,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.002056123958292787\n",
      "1 0.0020404768419831024\n",
      "2 0.0020250564320649566\n",
      "3 0.002009857924015696\n",
      "4 0.001994876646733686\n",
      "5 0.0019801080579702683\n",
      "6 0.001965547739947282\n",
      "7 0.0019511913951512907\n",
      "8 0.0019370348422964524\n",
      "9 0.0019230740124479978\n",
      "10 0.001909304945299319\n",
      "11 0.0018957237855951486\n",
      "12 0.0018823267796946328\n",
      "13 0.0018691102722676993\n",
      "14 0.0018560707031189828\n",
      "15 0.0018432046041333716\n",
      "16 0.0018305085963379896\n",
      "17 0.0018179793870754363\n",
      "18 0.0018056137672833098\n",
      "19 0.0017934086088756394\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for k in range(20):\n",
    "  \n",
    "  # forward pass\n",
    "  ypred = [n(x) for x in xs]\n",
    "  loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "  \n",
    "  # backward pass\n",
    "  for p in n.parameters():\n",
    "    p.grad = 0.0\n",
    "  loss.backward()\n",
    "  \n",
    "  # update\n",
    "  for p in n.parameters():\n",
    "    p.data += -0.1 * p.grad\n",
    "  \n",
    "  print(k, loss.data)\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 683,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Value(data=0.9817830812439714),\n",
       " Value(data=-0.9863881624765284),\n",
       " Value(data=-0.9766534529377958),\n",
       " Value(data=0.9729591216966093)]"
      ]
     },
     "execution_count": 683,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
